{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 12.3 Redis分布式爬虫实战"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 12.3.2 加入任务队列"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def push_redis_list():\n",
    "    r = Redis(host='137.189.204.65', port=6379 ,password='redisredis')\n",
    "    print (r.keys('*'))\n",
    "    \n",
    "    link_list = []\n",
    "    with open('alexa.txt', 'r') as file:\n",
    "        file_list = file.readlines()\n",
    "        for eachone in file_list:\n",
    "            link = eachone.split('\\t')[1]\n",
    "            link = link.replace('\\n','')\n",
    "            link_list.append(link)\n",
    "            if len(link_list) == 100:\n",
    "                break\n",
    "                \n",
    "    for url in link_list:\n",
    "        response = requests.get(url, headers=headers, timeout=20)\n",
    "        soup = BeautifulSoup(response.text, 'lxml')\n",
    "        img_list = soup.find_all('img')\n",
    "        for img in img_list:\n",
    "            img_url = img['src']\n",
    "            if img_url != '':\n",
    "                print (\"加入的图片url: \", img_url)\n",
    "                r.lpush('img_url',img_url)\n",
    "        print ('现在图片链接的个数为', r.llen('img_url'))\n",
    "    return"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 12.3.3 读取任务队列，下载图片"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_img():\n",
    "    r = Redis(host='137.189.204.65', port=6379 ,password='redisredis')\n",
    "    while True:\n",
    "        try:\n",
    "            url = r.lpop('img_url')\n",
    "            url = url.decode('ascii')\n",
    "            try:\n",
    "                response = requests.get(url, headers=headers,timeout = 20)\n",
    "                name = int(time.time())\n",
    "                f = open(str(name)+ url[-4:], 'wb')\n",
    "                f.write(response.content)\n",
    "                f.close()\n",
    "                print ('已经获取图片', url)\n",
    "            except Exception as e:\n",
    "                print ('爬取图片过程出问题', e)\n",
    "            time.sleep(3)\n",
    "        except Exception as e:\n",
    "            print (e)\n",
    "            time.sleep(10)\n",
    "            break\n",
    "    return "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 12.3.4 分布式爬虫代码"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "import time\n",
    "from redis import Redis\n",
    "headers={ 'User-Agent':'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36' }\n",
    "\n",
    "def push_redis_list():\n",
    "    #与上面此函数相同\n",
    "\n",
    "def get_img():\n",
    "    #与上面此函数相同\n",
    "\n",
    "if __name__ == '__main__':           \n",
    "    this_machine = 'master'          \n",
    "    print ('开始分布式爬虫')\n",
    "    if this_machine == 'master':\n",
    "        push_redis_list()\n",
    "    else:\n",
    "        get_img()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "import time\n",
    "from redis import Redis\n",
    "headers={ 'User-Agent':'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36' }\n",
    "\n",
    "def push_redis_list():\n",
    "    #与上面此函数相同\n",
    "\n",
    "def get_img():\n",
    "    #与上面此函数相同\n",
    "\n",
    "if __name__ == '__main__':           \n",
    "    this_machine = 'slave'          \n",
    "    print ('开始分布式爬虫')\n",
    "    if this_machine == 'master':\n",
    "        push_redis_list()\n",
    "    else:\n",
    "        get_img()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
